{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 看看 train & valid 之间共享 keyword 的比重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_path = \"/remote-home/weixionglin/vlp/Match/Preprocess/Frequency/csv_cache/test_keywords.csv\"\n",
    "valid_path = \"/remote-home/weixionglin/vlp/Match/Preprocess/Frequency/csv_cache/valid_keywords.csv\"\n",
    "train_path = \"/remote-home/weixionglin/vlp/Match/Preprocess/Frequency/csv_cache/train_keywords.csv\"\n",
    "\n",
    "df_test = pd.read_csv(test_path, sep=',')\n",
    "df_valid = pd.read_csv(valid_path, sep=',')\n",
    "df_train = pd.read_csv(train_path, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_words = set(df_test['word'].tolist())\n",
    "valid_words = set(df_valid['word'].tolist())\n",
    "train_words = set(df_train['word'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_inter_train: 8286\n",
      "10285\n",
      "32914\n"
     ]
    }
   ],
   "source": [
    "valid_inter_train = set.intersection(valid_words, train_words)\n",
    "print(f\"valid_inter_train: {len(valid_inter_train)}\")\n",
    "print(f\"{len(valid_words)}\")\n",
    "print(f\"{len(train_words)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7427 vs. 8286\n"
     ]
    }
   ],
   "source": [
    "# letters = [chr(i) for i in range(ord('a'),ord('z')+1)] + [chr(i) for i in range(ord('A'),ord('Z')+1)]\n",
    "inter_list = list(valid_inter_train)\n",
    "valid_inter_train = [word for word in inter_list if len(word) > 3]\n",
    "print(f\"{len(valid_inter_train)} vs. {len(inter_list)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 只在 caption 中留下包含 intersection 的单词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile('|'.join(valid_inter_train), re.IGNORECASE)\n",
    "\n",
    "# regex = re.compile('|'.join(list(valid_inter_train)[:5]), re.IGNORECASE)\n",
    "# regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "re.search(regex, 'cabbbbbbb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_disjoint(dataset: str, regex):\n",
    "    dir_path = Path(f\"/remote-home/share/medical/public/ROCO/{dataset}/radiology\")\n",
    "    csv_path = dir_path / f\"processed_{dataset}.csv\"\n",
    "    df_data = pd.read_csv(csv_path, sep=',')\n",
    "\n",
    "    pbar = tqdm(total=len(df_data))\n",
    "    for index, row in tqdm(df_data.iterrows(), desc='remove disjoint'):\n",
    "        # print(f\"\\033[42mPrev:\\033[0m {df_data.loc[index, 'caption']}\")\n",
    "        caption = row['caption']\n",
    "        words = caption.split(' ')\n",
    "        new_words = [word for word in words if re.search(regex, word)]\n",
    "        caption = ' '.join(new_words)\n",
    "        df_data.loc[index, 'caption'] = caption\n",
    "        # print(f\"\\033[42mPost:\\033[0m {df_data.loc[index, 'caption']}\")\n",
    "        # if index > 10:\n",
    "        #     raise ValueError(f\"index: {index}\")\n",
    "        pbar.update(1)\n",
    "    pbar.close()\n",
    "    return df_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remove disjoint: 8175it [01:19, 103.25it/s]9.92it/s]\n",
      "100%|██████████| 8175/8175 [01:19<00:00, 103.24it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset = 'valid'\n",
    "df_data = remove_disjoint(dataset=dataset, regex=regex)\n",
    "df_data.to_csv(f'/remote-home/weixionglin/vlp/Match/Preprocess/Frequency/csv_cache/{dataset}_disjoint.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remove disjoint: 65419it [12:29, 87.29it/s]82.51it/s] \n",
      "100%|██████████| 65419/65419 [12:29<00:00, 87.29it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset = 'train'\n",
    "df_data = remove_disjoint(dataset=dataset, regex=regex)\n",
    "df_data.to_csv(f'/remote-home/weixionglin/vlp/Match/Preprocess/Frequency/csv_cache/{dataset}_disjoint.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
